package com.mark.crawler.process.entity;

import lombok.Getter;
import lombok.Setter;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.Scheduler;

import java.util.ArrayList;
import java.util.List;

/**
 * @Description :  抓取组件
 * @Author :  Markburt
 * @CreateDate :  2020/3/7$ 下午 09:26$
 * @UpdateUser :  Markburt
 * @UpdateDate :  2020/3/7$ 下午 09:26$
 * @UpdateRemark :  Project Build
 * @Version :  1.0
 */
@Getter
@Setter
public class CrawlerComponent {

    /**
     * 页面处理类
     */
    private PageProcessor pageProcessor;
    /**
     * pipelineList 处理
     */
    private List<Pipeline> pipelineList = new ArrayList<Pipeline>();

    /**
     * 去重组件
     */
    private Scheduler scheduler;

    /**
     * 下载组件
     */
    private Downloader downloader;

    /**
     * 添加管道
     * @param pipeline
     */
    public void addPipeline(Pipeline pipeline) {
        pipelineList.add(pipeline);
    }
}
